{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lxml import etree\n",
    "import requests\n",
    "import time\n",
    "import pymongo\n",
    "\n",
    "from multiprocessing import Pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers={'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'}\n",
    "    \n",
    "def get_res(url):\n",
    "    response=requests.get(url,headers=headers)\n",
    "    return response.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_link():\n",
    "    url='https://cs.58.com/sale.shtml'\n",
    "    selector=etree.HTML(get_res(url))\n",
    "    \n",
    "    infos=selector.xpath('//*[@id=\"ymenu-side\"]/ul/li')\n",
    "    for info in infos:\n",
    "        hrefs= info.xpath ('ul/li/b/a/@href')\n",
    "        for href in hrefs:\n",
    "            print('https://cs.58.com/'+href)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://cs.58.com//shouji/\n",
      "https://cs.58.com//tongxunyw/\n",
      "https://cs.58.com//danche/\n",
      "https://cs.58.com//diandongche/\n",
      "https://cs.58.com//fzixingche/\n",
      "https://cs.58.com//sanlunche/\n",
      "https://cs.58.com//peijianzhuangbei/\n",
      "https://cs.58.com//diannao/\n",
      "https://cs.58.com//bijiben/\n",
      "https://cs.58.com//pbdn/\n",
      "https://cs.58.com//diannaopeijian/\n",
      "https://cs.58.com//zhoubianshebei/\n",
      "https://cs.58.com//shuma/\n",
      "https://cs.58.com//shumaxiangji/\n",
      "https://cs.58.com//mpsanmpsi/\n",
      "https://cs.58.com//youxiji/\n",
      "https://cs.58.com//ershoukongtiao/\n",
      "https://cs.58.com//dianshiji/\n",
      "https://cs.58.com//xiyiji/\n",
      "https://cs.58.com//bingxiang/\n",
      "https://cs.58.com//jiadian/\n",
      "https://cs.58.com//binggui/\n",
      "https://cs.58.com//chuang/\n",
      "https://cs.58.com//ershoujiaju/\n",
      "https://cs.58.com//yingyou/\n",
      "https://cs.58.com//yingeryongpin/\n",
      "https://cs.58.com//muyingweiyang/\n",
      "https://cs.58.com//muyingtongchuang/\n",
      "https://cs.58.com//yunfuyongpin/\n",
      "https://cs.58.com//fushi/\n",
      "https://cs.58.com//nanzhuang/\n",
      "https://cs.58.com//fsxiemao/\n",
      "https://cs.58.com//xiangbao/\n",
      "https://cs.58.com//meirong/\n",
      "https://cs.58.com//yishu/\n",
      "https://cs.58.com//shufahuihua/\n",
      "https://cs.58.com//zhubaoshipin/\n",
      "https://cs.58.com//yuqi/\n",
      "https://cs.58.com//tushu/\n",
      "https://cs.58.com//tushubook/\n",
      "https://cs.58.com//wenti/\n",
      "https://cs.58.com//yundongfushi/\n",
      "https://cs.58.com//jianshenqixie/\n",
      "https://cs.58.com//huju/\n",
      "https://cs.58.com//qiulei/\n",
      "https://cs.58.com//yueqi/\n",
      "https://cs.58.com//kaquan/\n",
      "https://cs.58.com//bangongshebei/\n",
      "https://cs.58.com//diannaohaocai/\n",
      "https://cs.58.com//bangongjiaju/\n",
      "https://cs.58.com//ershoushebei/\n",
      "https://cs.58.com//chengren/\n",
      "https://cs.58.com//nvyongpin/\n",
      "https://cs.58.com//qinglvqingqu/\n",
      "https://cs.58.com//qingquneiyi/\n",
      "https://cs.58.com//chengren/\n",
      "https://cs.58.com//xiaoyuan/\n",
      "https://cs.58.com//ershouqiugou/\n",
      "https://cs.58.com//tiaozao/\n",
      "https://cs.58.com//tiaozao/\n",
      "https://cs.58.com//tiaozao/\n"
     ]
    }
   ],
   "source": [
    "get_link()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "channel_list = '''\n",
    "    http://cs.58.com/shouji/\n",
    "    http://cs.58.com/tongxunyw/\n",
    "    http://cs.58.com/danche/\n",
    "    http://cs.58.com/fzixingche/\n",
    "    http://cs.58.com/diandongche/\n",
    "    http://cs.58.com/sanlunche/\n",
    "    http://cs.58.com/peijianzhuangbei/\n",
    "    http://cs.58.com/diannao/\n",
    "    http://cs.58.com/bijiben/\n",
    "    http://cs.58.com/pbdn/\n",
    "    http://cs.58.com/diannaopeijian/\n",
    "    http://cs.58.com/zhoubianshebei/\n",
    "    http://cs.58.com/shuma/\n",
    "    http://cs.58.com/shumaxiangji/\n",
    "    http://cs.58.com/mpsanmpsi/\n",
    "    http://cs.58.com/youxiji/\n",
    "    http://cs.58.com/jiadian/\n",
    "    http://cs.58.com/dianshiji/\n",
    "    http://cs.58.com/ershoukongtiao/\n",
    "    http://cs.58.com/xiyiji/\n",
    "    http://cs.58.com/bingxiang/\n",
    "    http://cs.58.com/binggui/\n",
    "    http://cs.58.com/chuang/\n",
    "    http://cs.58.com/ershoujiaju/\n",
    "    http://cs.58.com/bangongshebei/\n",
    "    http://cs.58.com/diannaohaocai/\n",
    "    http://cs.58.com/bangongjiaju/\n",
    "    http://cs.58.com/ershoushebei/\n",
    "    http://cs.58.com/yingyou/\n",
    "    http://cs.58.com/yingeryongpin/\n",
    "    http://cs.58.com/muyingweiyang/\n",
    "    http://cs.58.com/muyingtongchuang/\n",
    "    http://cs.58.com/yunfuyongpin/\n",
    "    http://cs.58.com/fushi/\n",
    "    http://cs.58.com/nanzhuang/\n",
    "    http://cs.58.com/fsxiemao/\n",
    "    http://cs.58.com/xiangbao/\n",
    "    http://cs.58.com/meirong/\n",
    "    http://cs.58.com/yishu/\n",
    "    http://cs.58.com/shufahuihua/\n",
    "    http://cs.58.com/zhubaoshipin/\n",
    "    http://cs.58.com/yuqi/\n",
    "    http://cs.58.com/tushu/\n",
    "    http://cs.58.com/tushubook/\n",
    "    http://cs.58.com/wenti/\n",
    "    http://cs.58.com/yundongfushi/\n",
    "    http://cs.58.com/jianshenqixie/\n",
    "    http://cs.58.com/huju/\n",
    "    http://cs.58.com/qiulei/\n",
    "    http://cs.58.com/yueqi/\n",
    "    http://cs.58.com/chengren/\n",
    "    http://cs.58.com/nvyongpin/\n",
    "    http://cs.58.com/qinglvqingqu/\n",
    "    http://cs.58.com/qingquneiyi/\n",
    "    http://cs.58.com/chengren/\n",
    "    http://cs.58.com/xiaoyuan/\n",
    "    http://cs.58.com/ershouqiugou/\n",
    "    http://cs.58.com/tiaozao/\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = pymongo.MongoClient('localhost', 27017,connect=False)\n",
    "mydb = client['mydb']\n",
    "tongcheng_url = mydb['tongcheng_url']\n",
    "tongcheng_info = mydb['tongcheng_info']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_links(channel,pages):\n",
    "    list_view = '{}pn{}/'.format(channel,str(pages))\n",
    "    try:\n",
    "        html = requests.get(list_view,headers=headers)\n",
    "        time.sleep(2)\n",
    "        selector = etree.HTML(html.text)\n",
    "        if selector.xpath('//tr'):\n",
    "            infos = selector.xpath('//tr')\n",
    "            for info in infos:\n",
    "                if info.xpath('td[2]/a/@href'):\n",
    "                    url = info.xpath('td[2]/a/@href')[0]\n",
    "                    tongcheng_url.insert_one({'url':url})\n",
    "                else:\n",
    "                    pass\n",
    "        else:\n",
    "            pass\n",
    "    except requests.exceptions.ConnectionError:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Process ForkPoolWorker-5:\n",
      "Process ForkPoolWorker-6:\n",
      "Process ForkPoolWorker-8:\n",
      "Process ForkPoolWorker-7:\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/pool.py\", line 108, in worker\n",
      "    task = get()\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/pool.py\", line 108, in worker\n",
      "    task = get()\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/process.py\", line 258, in _bootstrap\n",
      "    self.run()\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/pool.py\", line 108, in worker\n",
      "    task = get()\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/process.py\", line 93, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/queues.py\", line 334, in get\n",
      "    with self._rlock:\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/pool.py\", line 108, in worker\n",
      "    task = get()\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/synchronize.py\", line 96, in __enter__\n",
      "    return self._semlock.__enter__()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/queues.py\", line 335, in get\n",
      "    res = self._reader.recv_bytes()\n",
      "KeyboardInterrupt\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/connection.py\", line 216, in recv_bytes\n",
      "    buf = self._recv_bytes(maxlength)\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/connection.py\", line 407, in _recv_bytes\n",
      "    buf = self._recv(4)\n",
      "  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/connection.py\", line 379, in _recv\n",
      "    chunk = read(handle, remaining)\n",
      "KeyboardInterrupt\n"
     ]
    }
   ],
   "source": [
    "def get_all_links_from(channel):\n",
    "    for num in range(1,2):\n",
    "        get_links(channel,num)\n",
    "\n",
    "pool = Pool(processes=4)\n",
    "pool.map(get_all_links_from,channel_list.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_info(url):\n",
    "    html = requests.get(url,headers=headers)\n",
    "    selector = etree.HTML(html.text)\n",
    "    try:\n",
    "        title = selector.xpath('//h1/text()')[0]\n",
    "        if selector.xpath('//span[@class=\"price_now\"]/i/text()'):\n",
    "            price = selector.xpath('//span[@class=\"price_now\"]/i/text()')[0]\n",
    "        else:\n",
    "            price = \"无\"\n",
    "        if selector.xpath('//div[@class=\"palce_li\"]/span/i/text()'):\n",
    "            area = selector.xpath('//div[@class=\"palce_li\"]/span/i/text()')[0]\n",
    "        else:\n",
    "            area = \"无\"\n",
    "        view = selector.xpath('//p/span[1]/text()')[0]\n",
    "        if selector.xpath('//p/span[2]/text()'):\n",
    "            want = selector.xpath('//p/span[2]/text()')[0]\n",
    "        else:\n",
    "            want = \"无\"\n",
    "        info = {\n",
    "            'tittle':title,\n",
    "            'price':price,\n",
    "            'area':area,\n",
    "            'view':view,\n",
    "            'want':want,\n",
    "            'url':url\n",
    "        }\n",
    "        tongcheng_info.insert_one(info)\n",
    "    except IndexError:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/pymongo/topology.py:149: UserWarning: MongoClient opened before fork. Create MongoClient only after forking. See PyMongo's documentation for details: http://api.mongodb.org/python/current/faq.html#is-pymongo-fork-safe\n",
      "  \"MongoClient opened before fork. Create MongoClient only \"\n",
      "/usr/local/lib/python3.6/site-packages/pymongo/topology.py:149: UserWarning: MongoClient opened before fork. Create MongoClient only after forking. See PyMongo's documentation for details: http://api.mongodb.org/python/current/faq.html#is-pymongo-fork-safe\n",
      "  \"MongoClient opened before fork. Create MongoClient only \"\n",
      "/usr/local/lib/python3.6/site-packages/pymongo/topology.py:149: UserWarning: MongoClient opened before fork. Create MongoClient only after forking. See PyMongo's documentation for details: http://api.mongodb.org/python/current/faq.html#is-pymongo-fork-safe\n",
      "  \"MongoClient opened before fork. Create MongoClient only \"\n",
      "/usr/local/lib/python3.6/site-packages/pymongo/topology.py:149: UserWarning: MongoClient opened before fork. Create MongoClient only after forking. See PyMongo's documentation for details: http://api.mongodb.org/python/current/faq.html#is-pymongo-fork-safe\n",
      "  \"MongoClient opened before fork. Create MongoClient only \"\n"
     ]
    },
    {
     "ename": "MissingSchema",
     "evalue": "Invalid URL '//www.zhuanzhuan.com/detail.html?infoId=1086088364332728336&fullCate=5,36&fullLocal=414&metric=listpc&psid=194482777202870744119122549&entinfo=36835663580049_p&slot=-1': No schema supplied. Perhaps you meant http:////www.zhuanzhuan.com/detail.html?infoId=1086088364332728336&fullCate=5,36&fullLocal=414&metric=listpc&psid=194482777202870744119122549&entinfo=36835663580049_p&slot=-1?",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRemoteTraceback\u001b[0m                           Traceback (most recent call last)",
      "\u001b[0;31mRemoteTraceback\u001b[0m: \n\"\"\"\nTraceback (most recent call last):\n  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/pool.py\", line 119, in worker\n    result = (True, func(*args, **kwds))\n  File \"/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/pool.py\", line 44, in mapstar\n    return list(map(*args))\n  File \"<ipython-input-10-8447b2460f5b>\", line 2, in get_info\n    html = requests.get(url,headers=headers)\n  File \"/usr/local/lib/python3.6/site-packages/requests/api.py\", line 75, in get\n    return request('get', url, params=params, **kwargs)\n  File \"/usr/local/lib/python3.6/site-packages/requests/api.py\", line 60, in request\n    return session.request(method=method, url=url, **kwargs)\n  File \"/usr/local/lib/python3.6/site-packages/requests/sessions.py\", line 519, in request\n    prep = self.prepare_request(req)\n  File \"/usr/local/lib/python3.6/site-packages/requests/sessions.py\", line 462, in prepare_request\n    hooks=merge_hooks(request.hooks, self.hooks),\n  File \"/usr/local/lib/python3.6/site-packages/requests/models.py\", line 313, in prepare\n    self.prepare_url(url, params)\n  File \"/usr/local/lib/python3.6/site-packages/requests/models.py\", line 387, in prepare_url\n    raise MissingSchema(error)\nrequests.exceptions.MissingSchema: Invalid URL '//www.zhuanzhuan.com/detail.html?infoId=1086088364332728336&fullCate=5,36&fullLocal=414&metric=listpc&psid=194482777202870744119122549&entinfo=36835663580049_p&slot=-1': No schema supplied. Perhaps you meant http:////www.zhuanzhuan.com/detail.html?infoId=1086088364332728336&fullCate=5,36&fullLocal=414&metric=listpc&psid=194482777202870744119122549&entinfo=36835663580049_p&slot=-1?\n\"\"\"",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mMissingSchema\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-b47e1d492aad>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mpool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPool\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprocesses\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mget_info\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrest_urls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/pool.py\u001b[0m in \u001b[0;36mmap\u001b[0;34m(self, func, iterable, chunksize)\u001b[0m\n\u001b[1;32m    264\u001b[0m         \u001b[0;32min\u001b[0m \u001b[0ma\u001b[0m \u001b[0mlist\u001b[0m \u001b[0mthat\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mreturned\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    265\u001b[0m         '''\n\u001b[0;32m--> 266\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_map_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmapstar\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    267\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mstarmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/Cellar/python/3.6.4_4/Frameworks/Python.framework/Versions/3.6/lib/python3.6/multiprocessing/pool.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    642\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    643\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 644\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    645\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    646\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_set\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mMissingSchema\u001b[0m: Invalid URL '//www.zhuanzhuan.com/detail.html?infoId=1086088364332728336&fullCate=5,36&fullLocal=414&metric=listpc&psid=194482777202870744119122549&entinfo=36835663580049_p&slot=-1': No schema supplied. Perhaps you meant http:////www.zhuanzhuan.com/detail.html?infoId=1086088364332728336&fullCate=5,36&fullLocal=414&metric=listpc&psid=194482777202870744119122549&entinfo=36835663580049_p&slot=-1?"
     ]
    }
   ],
   "source": [
    "db_urls = [item['url'] for item in tongcheng_url.find()]\n",
    "db_infos = [item['url'] for item in tongcheng_info.find()]\n",
    "x = set(db_urls)\n",
    "y = set(db_infos)\n",
    "rest_urls = x - y\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    pool = Pool(processes=4)\n",
    "    pool.map(get_info,rest_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
